package hudong;

import org.apache.commons.io.FileUtils;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import util.CosineSimilarityAlgorithm;

import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.net.URISyntaxException;
import java.net.URLEncoder;
import java.util.*;
import java.util.concurrent.Executor;

/**
 * Created by zhongwei on 2017/1/5.
 */
public class Entry extends Thread {
    private static final String QUERY = "http://www.baike.com/wiki/[KEYWORD]";
    static FileWriter fileWriter = null;
    static Set<String> set = new HashSet<>();
    private static HttpClient client;
    private static RequestConfig requestConfig = null;

    @Override
    public void run() {
        try {
            main(null);
        } catch (URISyntaxException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
    }
    public static void main(String[] args) throws URISyntaxException, IOException, InterruptedException {
        client = HttpClients.custom().setUserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.95 Safari/537.36").build();
        requestConfig = RequestConfig.custom().setConnectTimeout(5000).setSocketTimeout(5000).build();
        File result_file = new File("hudong_result.txt");
        fileWriter = new FileWriter(result_file, true);
        String content = FileUtils.readFileToString(result_file);
        String[] dones = content.split("==========\n");
        for (String str : dones) {
            if (str.trim().length() > 0) {
                set.add(str.split("\n")[0]);
            }
        }
        List<String> peoples = FileUtils.readLines(new File("peoples"));
        int i = set.size();
        int total = peoples.size();
        for (String str : peoples) {
            if (set.contains(str))
                continue;
            System.out.println("hudong process : " + i++ + " / " + total);
            String[] arr = str.split("\t");
            if (arr.length == 2) {
                String people = arr[0];
                String url = QUERY.replace("[KEYWORD]", URLEncoder.encode(people));
                String page1 = download(url);
                judge(page1, str);
            }
//            sleep(1000);
        }
        System.out.println("hudong all is done");
        fileWriter.flush();
        fileWriter.close();
    }

    private static String download(String url) {
        try {
            HttpGet get = new HttpGet(url);
            get.setConfig(requestConfig);
            HttpResponse response = client.execute(get);
            String content = EntityUtils.toString(response.getEntity(), "UTF-8");
            EntityUtils.consumeQuietly(response.getEntity());
            return content;
        } catch (Exception e) {
            return download(url);
        }
    }

    private static void judge(String page1, String people) throws IOException {
        if (page1.contains("囧，互动百科尚未收录")) {
            // 没有找到词条就不用进一步处理
            fileWriter.write(people + "\n\n");
            fileWriter.write("==========\n");
            fileWriter.flush();
        } else if (page1.contains("是个多义词，全部含义如下")) {
            // 多义词，需要根据公司名称进一步过滤
            processMultiSemantic(page1, people);
        } else {
            processDetail(page1, people);
        }
    }

    private static void processMultiSemantic(String page1, String people) throws IOException {
        Document document = Jsoup.parse(page1);
        Elements lis = null;
        try {
            lis = document.getElementById("polysemyAll").getElementsByTag("li");
        } catch (Exception e) {
            e.printStackTrace();
        }
        final String[] arr = people.split("\t");
        Map<String, String> map = new HashMap<>();
        for (Element element : lis) {
            try {
                if (element.text().trim().length() > 0)
                    map.put(element.text(), element.getElementsByTag("a").get(0).attr("href"));
            } catch (IndexOutOfBoundsException e) {
                // ignore
            }
        }
        List<Map.Entry<String, String>> list = new ArrayList<>(map.entrySet());
        Collections.sort(list, new Comparator<Map.Entry<String, String>>() {
            @Override
            public int compare(Map.Entry<String, String> o1, Map.Entry<String, String> o2) {
                return CosineSimilarityAlgorithm.getSimilarity(o2.getKey(), arr[1]).compareTo(CosineSimilarityAlgorithm.getSimilarity(o1.getKey(), arr[1]));
            }
        });
        double maxSim = 0d;
        try {
            maxSim = CosineSimilarityAlgorithm.getSimilarity(list.get(0).getKey(), arr[1]);
        } catch (Exception e) {

        }
        if (maxSim < 0.2d) {
            fileWriter.write(people + "\n\n");
            fileWriter.write("==========\n");
            fileWriter.flush();
        } else {
            try {
                String url = list.get(0).getValue();
                String detailPage = download(url);
                processDetail(detailPage, people);
            } catch (Exception e) {
                // 已经选中目标记录
                processDetail(page1, people);
            }
        }
    }

    private static void processDetail(String page1, String people) throws IOException {
        Document document = Jsoup.parse(page1);
        try {
            Elements eles = document.getElementById("datamodule").getElementsByTag("tr");
            fileWriter.write(people + "\n");
            int size = eles.size();
            for (int i = 0; i < size; i++) {
                Elements tds = eles.get(i).getElementsByTag("td");
                for (Element td : tds) {
                    fileWriter.append(td.text() + "\t\t");
                }
            }
            fileWriter.write("\n==========\n");
            fileWriter.flush();
        } catch (Exception e) {
            // 没有基本信息一栏
            fileWriter.write(people + "\n\n");
            fileWriter.write("==========\n");
            fileWriter.flush();
        }
    }

}
